/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package com.elookinto.blackhat;

import com.gargoylesoftware.htmlunit.ProxyConfig;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class LinkedInUser extends WebCrawler {

    static int number = 0;
    private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g"
            + "|png|tiff?|mid|mp2|mp3|mp4"
            + "|wav|avi|mov|mpeg|ram|m4v|pdf"
            + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");
    /**
     * You should implement this function to specify whether the given url
     * should be crawled or not (based on your crawling logic).
     */
    static final int USER_PROFILE_SLASH_COUNT = 7;
    static final int MAX_RETURN = 10000;
    final static WebClient webClient = new WebClient();
     static {
         ProxyConfig proxyConfig = new ProxyConfig();
         proxyConfig.setProxyHost("proxy.entp.attws.com");
         proxyConfig.setProxyPort(8080);
         webClient.getOptions().setProxyConfig(proxyConfig);
         webClient.getOptions().setJavaScriptEnabled(false);
     }
    @Override
    public boolean shouldVisit(WebURL url) {
        String href = url.getURL().toLowerCase();
        System.out.println(url);
        int slashCount = 0;
        for (int k = 0; k < href.length(); k++) {
            if (href.charAt(k) == '/') {
                slashCount++;
            }
        }

        boolean b = !FILTERS.matcher(href).matches() && (href.startsWith("http://www.linkedin.com/pub")
                || href.startsWith("http://www.linkedin.com/directory")) && slashCount < USER_PROFILE_SLASH_COUNT;
        if (slashCount == USER_PROFILE_SLASH_COUNT) {
            try {
                HtmlPage hp = webClient.getPage(href);
                String text = hp.asText();
                Matcher m = p.matcher(text);
                if (m.find()) {
                    
                    int count = Integer.parseInt(m.group(0).split("\\D")[0]);
                     System.out.println(m.group(0) +"____"+href+ "_____" + count);
                    if (count > 15) {
                        insert(href, count);
                    }
                } else {
                    System.out.println("Match not found");
                }

            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        /*if (b) {
         number++;
         if (number > MAX_RETURN) {
         System.exit(0);
         }
         }*/
        return b;
    }
    Pattern p = Pattern.compile("\\d+\\W+connection");

    /**
     * This function is called when a page is fetched and ready to be processed
     * by your program.
     */
    public void insert(String url, int count) {
        Connection con = null;
        try {
            con = Util.getCon();

            PreparedStatement st = con.prepareCall("insert into linkedin (url, id, insert_time, conn_count ) values (?, null, current_timestamp, ?)");
            st.setString(1, url);
            st.setInt(2, count);
            st.execute();
            number++;
            if (number > MAX_RETURN) {
                System.exit(0);
            }
            st.close();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                con.close();
            } catch (Exception e) {
            }
        }
    }

    public void visit(Page page) {
        String url = page.getWebURL().getURL();
        System.out.println("URL: " + url);

        int slashCount = 0;
        for (int k = 0; k < url.length(); k++) {
            if (url.charAt(k) == '/') {
                slashCount++;
            }
        }
        if (slashCount == 7 && page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            String text = htmlParseData.getText();
            Matcher m = p.matcher(text);
            if (m.find()) {
                System.out.append(m.group(0));
                int count = Integer.parseInt(m.group(0).split("\\D")[0]);
                if (count > 15) {
                    insert(url, count);
                }
            } else {
                System.out.println("Match not found");
            }

            /*String html = htmlParseData.getHtml();
             List<WebURL> links = htmlParseData.getOutgoingUrls();

             System.out.println("Text length: " + text.length());
             System.out.println("Html length: " + html.length());
             System.out.println("Number of outgoing links: " + links.size());*/
        }



    }
}